package com.example.myFirstWebmagic.company;

import java.util.ArrayList;
import java.util.List;

import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;

import com.example.myFirstWebmagic.company.entity.CompanyJcbg;
import com.example.myFirstWebmagic.pipeline.ConsolePipelineDemo;
import com.example.myFirstWebmagic.util.CastUtil;
import com.example.myFirstWebmagic.util.StringUtil;
import com.google.gson.Gson;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.pipeline.FilePipeline;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Html;

/**
 * 
 * ClassName: CompanyBgProcess 
 * @Description: 企业年度监测报告单独爬取
 * @author xiongyao
 * @date 2017年3月29日
 */

@Component("companyBgProcess")
public class CompanyBgProcess implements PageProcessor{

private static final Logger LOGGER=LoggerFactory.getLogger(CompanyProcess.class);
	
	private Site site = Site.me().setRetryTimes(10).setSleepTime(10000);
	
	//翻页查询
	private static final String QUERY_URL="http://59.172.182.106/qyjc/jcjgXxOne.rh?pageNumber=";
	
	List<String> targetReuqest=new ArrayList<String>();
	
	//环保厅的首页
	private static final String HOME_URL="http://59.172.182.106/qyjc/jcjgXxOne.rh";
	
	List<String> companyUrlList=new ArrayList<String>();
	
	//查询企业监测首页
	private static final String COMPANY_URL="http://59.172.182.106/qyjc/jcjgXxTwo.rh?id=";
	
	//查询自行监测年度报告
	private static final String NDBGDETAIL="http://59.172.182.106/qyjc/ndbgDetail.rh?";
	
	@Override
	public void process(Page page) {
		// TODO Auto-generated method stub
		//如果是在平台主页获取企业主页链接地址
		Html html = page.getHtml();
		if(page.getRequest().getUrl().equals(HOME_URL)||page.getRequest().getUrl().startsWith(QUERY_URL)){
			String pageNumString=html.xpath("//*[@id='queryForm']/table[2]/tbody/tr/td/table[1]/tbody/tr/td/table[3]/tbody/tr[2]/td/table/tbody/tr[1]/td[2]/table[3]/tbody/tr/td/div/table/tbody/tr/td/text()").get();
			int pageNum;
			if(StringUtil.isNotEmpty(pageNumString)){
				pageNum=CastUtil.castInt(pageNumString.substring(pageNumString.lastIndexOf("共")+1, pageNumString.indexOf("页")).trim()) ;
				for(int i=2;i<=pageNum;i++){
					targetReuqest.add(QUERY_URL+i);
				}
			}
			companyUrlList=html.xpath("*[@id='queryForm']/table[2]/tbody/tr/td/table[1]/tbody/tr/td/table[3]/tbody/tr[2]/td/table/tbody/tr[1]/td[2]/table[2]/tbody/tr[1]/td[1]/table[1]/tbody//tr/td[1]/div/a/@href").all();
			List<String> targetUrl=new ArrayList<String>();
			targetUrl.addAll(targetReuqest);
			targetUrl.addAll(companyUrlList);
			page.addTargetRequests(targetUrl);
		}
		//如果是企业主页
		if (page.getRequest().getUrl().startsWith(COMPANY_URL)) {
		
			/**
			 * 企业年度监测报告单独爬取
			 */
			
			//企业自行监测年度报告
			List<String> idsAndYears= html.xpath("//*[@id='link_info4']/table/tbody/tr/td/table/tbody//a/@id").all();
			
			List<String> zxjcBg=new ArrayList<String>();
			for(String idAndYear:idsAndYears){
				String[] strings= StringUtil.split(idAndYear,"_");
				String qyId=strings[1];
				String nd=strings[2];
				String ndbcDetailUrl=NDBGDETAIL+"qyId="+qyId+"&nd="+nd;
				zxjcBg.add(ndbcDetailUrl);
			}
			page.addTargetRequests(zxjcBg);
		}
		
		//年度监测报告
		if(page.getRequest().getUrl().startsWith(NDBGDETAIL)){
			Gson gson=new Gson();
			CompanyJcbg jcbg=gson.fromJson(page.getJson().toString(), CompanyJcbg.class);
			LOGGER.info(jcbg.toString());
			page.putField("jcbg", jcbg);
		}
	}

	@Test
	public void testProcess(){
		String [] startUrl=new String[]{"http://59.172.182.106/qyjc/jcjgXxOne.rh"};
		// TODO Auto-generated method stub
		Spider.create(new CompanyBgProcess())
        //从"http://datacenter.mep.gov.cn/index!getCityMain.action"开始抓
        .addUrl(startUrl)
        .addPipeline(new ConsolePipelineDemo()).addPipeline(new FilePipeline("C:/Users/Administrator/Desktop/临时目录/company.txt"))
        //开启5个线程抓取
        .thread(5)
        //启动爬虫
        .run();
	}
	
	
	@Override
	public Site getSite() {
		// TODO Auto-generated method stub
		return this.site;
	}

}
